# Author    : CoffeeChicken
# Date      : 2020-10-10 20:49
# Function  : 演示 爬取sexy link

import requests
from lxml import etree

url_path = "http://mmhy001.me/?forum-7-{num}.htm"
headers = {
    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) "
                  "Chrome/85.0.4183.121 Safari/537.36"
}
front_url = "http://mmhy001.me/"

for i in range(10):
    url = url_path.format(num=int(i + 1))
    response_front = requests.get(url, headers).text
    # 加载到etree对象中
    tree_front = etree.HTML(response_front)
    link_list = tree_front.xpath("//div[@class='card-body']/ul/li/@data-href")

    for links in link_list:
        page_link = front_url + links
        response_page = requests.get(page_link, headers).text
        tree_page = etree.HTML(response_page)
        title = tree_page.xpath("//h5[@class='break-all']/text()")[0].strip()

        one_link = tree_page.xpath("//div[@class='card-body']/div[@class='message break-all']/p/a/text()")

        save_str = title + "\n\n" + one_link.__str__() + "\n\n\n\n\n"

        with open("links.txt", 'a') as fp:
            fp.write(save_str)

        # print(one_link)
